const fs = require('fs');
const fastCsv = require('fast-csv');

// 使用fast-csv库处理大量CSV数据
const records = [];
const startTime = Date.now();

fs.createReadStream('../521.CSV')
  .pipe(fastCsv.parse({ 
    headers: true,
    trim: true,
    ignoreEmpty: true,
    discardUnmappedColumns: true,
    quote: '"',
    escape: '\\',
    skipLines: 0,
    maxRows: 0 // 设置为0表示不限制行数
  }))
  .on('error', error => {
    console.error('解析错误:', error);
  })
  .on('data', row => {
    records.push(row);
    // 每处理10000行输出一次进度
    if (records.length % 10000 === 0) {
      console.log(`已处理 ${records.length} 行数据...`);
    }
  })
  .on('end', rowCount => {
    const endTime = Date.now();
    const duration = (endTime - startTime) / 1000;
    
    console.log(`成功解析记录数: ${records.length}`);
    console.log(`解析耗时: ${duration} 秒`);
    console.log('前3条记录示例:');
    console.log(records.slice(0, 3));
  });